package com.example.IK_DF_IDF_Demo;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;

import org.wltea.analyzer.core.IKSegmenter;
import org.wltea.analyzer.core.Lexeme;

/**
 * Created on 2018-07-31
 * @author fenghongyu
 */
public class TFIDFMeasure {
    /**
     * @param args
     */
    private static List<String> fileList = new ArrayList<>();

    public static List<String> readDirs(String filepath){
        try {
            File file = new File(filepath);
            if (!file.isDirectory()) {
                System.out.println("输入的[]");
                System.out.println("filepath:" + file.getAbsolutePath());
            } else {
                String[] flist = file.list();
                for (int i = 0; i < flist.length; i++) {
                    File newfile = new File(filepath + "\\" + flist[i]);
                    if (!newfile.isDirectory()) {
                        fileList.add(newfile.getAbsolutePath());
                    } else if (newfile.isDirectory()) {
                        readDirs(filepath + "\\" + flist[i]);
                    }
                }
            }
        } catch (Exception e) {
            System.out.println(e.getMessage());
        }
        return fileList;
    }

    public static String readFile(String file) throws  IOException {
        StringBuffer strSb = new StringBuffer();
        InputStreamReader inStrR = new InputStreamReader(new FileInputStream(
                file), "UTF-8");
        BufferedReader br = new BufferedReader(inStrR);
        String line = br.readLine();
        while (line != null) {
            strSb.append(line).append("\r\n");
            line = br.readLine();
        }
        br.close();
        return strSb.toString();
    }

    @SuppressWarnings("resource")
    public static List<String> cutWords(String file) throws IOException {
        String text = TFIDFMeasure.readFile(file);
        List<String> results = new ArrayList<>();
        String result = "";
        if(text == null){  //如果str为空，返回空值
            return results;
        }
        try{
            StringReader sr = new StringReader(text); //直接使用 StringReader 读取字符串
            IKSegmenter ik = new IKSegmenter(sr, true); //true智能切分
            Lexeme lex = null;
            while((lex=ik.next())!=null){
                results.add(lex.getLexemeText()+"| ");
            }

        }catch(IOException e){
            e.printStackTrace();
        }
        return results;
    }

    public static Map<String, Integer> normalTF(List<String> cutwords) {
        Map<String, Integer> resTF = new HashMap<String, Integer>();

        for (String word : cutwords) {
            if (resTF.get(word) == null) {
                resTF.put(word, 1);
            } else {
                resTF.put(word, resTF.get(word) + 1);
            }
        }
        return resTF;
    }

    public static Map<String, Double> getTF(List<String> cutwords) {
        Map<String, Double> resTF = new HashMap<String, Double>();

        int wordLen = cutwords.size();
        Map<String, Integer> intTF = TFIDFMeasure.normalTF(cutwords);

        for (Entry<String, Integer> entry : intTF.entrySet()) {
            resTF.put(entry.getKey().toString(),
                    Double.parseDouble(entry.getValue().toString()) / wordLen);
        }
        return resTF;
    }

    public static Map<String, Map<String, Integer>> normalTFAllFiles(String dirc) throws IOException {
        Map<String, Map<String, Integer>> allNormalTF = new HashMap<>();

        List<String> filelist = TFIDFMeasure.readDirs(dirc);
        for (String file : filelist) {
            List<String> cutwords = TFIDFMeasure.cutWords(file);
            Map<String, Integer> dict = TFIDFMeasure.normalTF(cutwords);
            allNormalTF.put(file, dict);
        }
        return allNormalTF;
    }

    public static Map<String, Map<String, Double>> getFilesTF(String dirc) throws IOException {
        Map<String, Map<String, Double>> filesTF = new HashMap<>();
        List<String> filelist = TFIDFMeasure.readDirs(dirc);
        for (String file : filelist) {
            List<String> cutwords = TFIDFMeasure.cutWords(file);
            Map<String, Double> dict = TFIDFMeasure.getTF(cutwords);
            filesTF.put(file, dict);
        }
        return filesTF;
    }

    public static Map<String, Double> getFilesIDFs(
        Map<String, Map<String, Double>> all_tf) {
        Map<String, Double> resIdf = new HashMap<>();
        Map<String, Integer> dict = new HashMap<>();
        int docNum = fileList.size();
        for (int i = 0; i < docNum; i++) {
            Map<String, Double> temp = all_tf.get(fileList.get(i));
            for (Entry<String, Double> entry : temp.entrySet()) {
                String word = entry.getKey();
                if (dict.get(word) == null) {
                    dict.put(word, 1);
                } else {
                    dict.put(word, dict.get(word) + 1);
                }
            }
        }
        System.out.println("IDF for every word is:");
        for (Entry<String, Integer> entry : dict.entrySet()) {
            //加1 用来做平滑，避免生僻词在语料库中没有，则分母为0
            double value = (float) Math.log(docNum / Double.parseDouble(entry.getValue().toString())) + 1;
            resIdf.put(entry.getKey(), value);
        }
        return resIdf;
    }

    public static Map<String, Map<String, Double>> getFilesTF_IDF(Map<String, Map<String, Double>> filesTF, Map<String, Double> filesIDFs) {
        Map<String, Map<String, Double>> filesTF_IDF = new HashMap<>();
        int docNum = fileList.size();
        for (int i = 0; i < docNum; i++) {
            String filepath = fileList.get(i);
            Map<String, Double> tfidf = new HashMap<>();
            Map<String, Double> temp = filesTF.get(filepath);
            for (Entry<String, Double> entry : temp.entrySet()) {
                String word = entry.getKey();
                Double value = entry.getValue() * filesIDFs.get(word);
                tfidf.put(word, value);
            }
            filesTF_IDF.put(filepath, tfidf);
        }
        return filesTF_IDF;
    }

    public static void disTfIdf(Map<String, Map<String, Double>> tfidf){

        System.out.println("TF-IDF for Every file is :");
        Map<String,Double> resultMap = new HashMap<>();
        for (Entry<String, Map<String, Double>> entrys : tfidf.entrySet()) {
            System.out.println("FileName: " + entrys.getKey());
            System.out.println("{");
            Map<String, Double> temp = entrys.getValue();
            ArrayList<Entry<String, Double>> infoIds = new ArrayList<>(temp.entrySet());
            Collections.sort(infoIds, (o1, o2) -> o2.getValue().compareTo(o1.getValue()));
            int i = 0;
            for (Entry<String, Double> entry : infoIds) {
                System.out.println(entry.getKey() + " = " + entry.getValue().toString() + ", ");
                i++;
                if(i>5){
                    break;
                }
            }
            System.out.println("}");
            resultMap.put(entrys.getKey(), infoIds.get(0).getValue()+infoIds.get(1).getValue()
                    );//存入高分中前6名的和
        }
        ArrayList<Entry<String, Double>> infoIds2 = new ArrayList<>(resultMap.entrySet());
        Collections.sort(infoIds2, (o1, o2) -> o2.getValue().compareTo(o1.getValue()));
        for (Entry<String, Double> entry : infoIds2) {
            System.out.println(entry.getKey() + " = " + entry.getValue().toString() + ", ");
        }
    }

    public static void main(String[] args) throws IOException {
        String file = TFIDFMeasure.class.getResource("/data").getPath();
//        System.out.println(jarPath);
//        String file = "classPath://data";
        Map<String, Map<String, Double>> filesTF = getFilesTF(file);
        Map<String, Double> filesIDFs = getFilesIDFs(filesTF);
        Map<String, Map<String, Double>> filesTF_IDF = getFilesTF_IDF(filesTF, filesIDFs);
        disTfIdf(filesTF_IDF);
    }
    //https://blog.csdn.net/songzhilian22/article/details/49636725
}
